# !/usr/bin/python
# -*- coding: utf-8 -*-
import json
from util import loadJson, get_data, split, black_list, generate_dict, insert_data


def filter():
    dictionary = loadJson('./dict.json')
    data = get_data()

    for item in data:

        if len(item[2]) > 30:
            continue

        df_score = 0

        id = str(item[0])
        img_urls = json.loads(item[1])
        context = black_list(item[2].encode('utf-8'))

        words = split(context)
        for word in words:
            try:
                df_score += dictionary[word.decode('utf-8')]
            except KeyError:
                pass

        if len(context) == 0 or df_score == 0:
            continue

        print id, df_score
        print context

        for idx, url in enumerate(img_urls):
            if url[-4:] == '.jpg':
                img_id = id + '_' + str(idx)
                insert_data(img_id, context, df_score, url)


filter()
